package com.otaku.crawler;

import java.util.List;

import org.apache.http.util.ExceptionUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import com.otaku.crawler.model.NASaveMoneyPojo;
import com.otaku.crawler.model.WAHTPojo;
import com.otaku.crawler.parser.NASaveMoneyParser;
import com.otaku.crawler.parser.WAHTParser;
import com.otaku.crawler.util.Constants;
import com.otaku.crawler.util.EmailHelper;
import com.otaku.crawler.util.ExceptionUtil;

import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;

public class WAHTCrawler extends WebCrawler{
	private static final Logger log = Logger.getRootLogger();
	
	public boolean shouldVisit(WebURL url) {
		String href = url.getURL().toLowerCase();
		return !Constants.FILTERS.matcher(href).matches()
				&& href.startsWith("http://www.woaihaitao.com")
				&& !href.startsWith("http://www.woaihaitao.com/tag/") //内容建设中
				&& !href.startsWith("http://www.woaihaitao.com/feed/") //XML
				&& !href.startsWith("http://www.woaihaitao.com/forums/") //论坛
				&& !href.startsWith("http://www.woaihaitao.com/?p=")
				&& !href.endsWith(".html")
				&& !href.endsWith("/feed")
				&& !href.contains(".css?ver=");
	}

	public void visit(Page page) {
		String url = page.getWebURL().getURL();
		try {
			if (page.getParseData() instanceof HtmlParseData) {
				HtmlParseData htmlParseData = (HtmlParseData) page
						.getParseData();
				String html = htmlParseData.getHtml();
				Document d = Jsoup.parse(html);

				List<WAHTPojo> l = WAHTParser.parse(d);

				log.info("url:"+url+"\nget list size:"+l.size());
			}
		}catch (Exception e) {
			EmailHelper.sendMsg("WAHTCrawler caught exception in url:\n"+url,e);
			e.printStackTrace();
			return;
		}
	}
}
